package com.example.demo_pc.util;

import com.alibaba.fastjson.JSON;
import com.example.demo_pc.vo.ExceptionHandVo;
import com.example.demo_pc.vo.ResultVo;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @Description :
 * @Author : wanghao
 * @Date: 2020-11-30 10:16
 */
public class Reptiles implements PageProcessor {

    private static final Logger logger = LoggerFactory.getLogger(Reptiles.class);

    //页面URL的正则表达式
    //.是匹配所有的字符，//.表示只匹配一个，//.?同理
    private static String REGEX_PAGE_URL_1 = "https://zh.nyahentai.xyz/parody/[A-Za-z\\-]+/";
    private static String REGEX_PAGE_URL_2 = "https://zh.nyahentai.xyz/parody/[A-Za-z\\-]+/page/\\d+";
    private static String REGEX_PAGE_URL_INFO = "https://zh.nyahentai.xyz/g/\\d+/";
    private static String REGEX_PAGE_URL_INFO_LIST = "https://zh.nyahentai.xyz/g/\\d+/list2/";
    private static String REGEX_PAGE_URL_INFO_PAGE = "https://zh.nyahentai.xyz/g/\\d+/list/\\d+";
    //https://zh.nyahentai.xyz/g/337104/list/1/   https://zh.nyahentai.xyz/g/335023/list/1/
    public static int PAGE_SIZE = 311;
    // public static String FILE_OUT_PATH = "D:\\爬虫\\艦隊\\";
    public static String FILE_OUT_PATH = "/usr/local/photo/爬虫/艦隊/";


    //下载张数
    public Map<String,String> nameMap = new HashMap<>();
    public List<String> errorList=new ArrayList<>();
    public List<String> existList=new ArrayList<>();
    public List<String> downLoadList=new ArrayList<>();
    public List<String> pageErrorList=new ArrayList<>();
    public String fileOutPath;
    public int num;

    public List<String> getExistList() {
        return existList;
    }

    public void setExistList(List<String> existList) {
        this.existList = existList;
    }

    public List<String> getErrorList() {
        return errorList;
    }

    public void setErrorList(List<String> errorList) {
        this.errorList = errorList;
    }

    public String getFileOutPath() {
        return fileOutPath;
    }

    public void setFileOutPath(String fileOutPath) {
        this.fileOutPath = fileOutPath;
    }

    public Reptiles(String fileOutPath, int num) {
        this.fileOutPath = fileOutPath;
        this.num = num;
    }

    public List<String> getDownLoadList() {
        return downLoadList;
    }

    public void setDownLoadList(List<String> downLoadList) {
        this.downLoadList = downLoadList;
    }

    public Reptiles(List<String> pageErrorList) {
        this.pageErrorList = pageErrorList;
    }

    public int getNum() {
        return num;
    }

    public void setNum(int num) {
        this.num = num;
    }

    private Site site = Site.me().setRetryTimes(3).setSleepTime(3000).setTimeOut(8000).setCycleRetryTimes(3)
            .addHeader("Accept-Encoding", "/").setUserAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36)");
    //配置
    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {

        //是图片列表页面
        if(page.getUrl().regex(REGEX_PAGE_URL_1).match() || page.getUrl().regex(REGEX_PAGE_URL_2).match()){
            Html html = page.getHtml();
            List<String> all = html.xpath("//div[@class='container']/div[@class='gallery']/a").all();
            for (String s : all) {
                Document parse = Jsoup.parse(s);
                String attr = parse.getElementsByTag("a").attr("href");
                page.addTargetRequest("https://zh.nyahentai.xyz"+attr);
            }
            logger.info("一共 " + all.size()+" 个文件");
            return;
        }else if(page.getUrl().regex(REGEX_PAGE_URL_INFO_LIST).match()){

            String urlHtml = page.getUrl().toString();
            System.out.println("图片页当前url:"+urlHtml);
            Html html = page.getHtml();
            List<String> imgList= html.xpath("//section[@id='image-container']/img").all();
            String key = getContent(REGEX_PAGE_URL_INFO, urlHtml);
            String folder = nameMap.get(key);
            String path=fileOutPath+"/"+folder;
            File file=new File(path);
            if (file.exists()){
                existList.add("当前url已存在: "+key+" 文件名: "+folder+" 地址: "+path);
                downLoadList.add(folder+" 当前文件已存在");
                logger.info("当前url已存在: "+key+" 文件名: "+folder+" 地址: "+path);
                return;
            }
            downLoadList.add(folder);
            long startTime = System.currentTimeMillis();    //获取开始时间
            MultiThreadUtils<String> stringMultiThreadUtils = new MultiThreadUtils<>();
            stringMultiThreadUtils.setCountNum(num);
            try {
                stringMultiThreadUtils.execute(imgList, new AbstractTask<String>() {
                    @Override
                    public void execute(List<String> data) throws Exception {
                        for (String file : data) {
                            Document doc = Jsoup.parse(file);
                            String attrJPG = doc.getElementsByTag("img").attr("data-src");
                            String downPath=attrJPG.substring(0,attrJPG.lastIndexOf("."));
                            String fileName=attrJPG.substring(attrJPG.lastIndexOf("/"));
                            fileName=getContent("\\d+",fileName);
                            fileName = FileUtils.numToString(fileName, 3);
                            int flag=0;
                            long downStartTime = System.currentTimeMillis();    //获取开始时间
                            //获取开始时间
                            for (int i =1 ; i<=15;i++){
                                boolean res = DownloadImage.imgDownLoad(downPath+".jpg" , fileName + ".jpg", path,i);
                                if (res){
                                    flag++;
                                    break;
                                }
                                boolean result = DownloadImage.imgDownLoad(downPath+".png" , fileName + ".jpg", path,i);
                                if (result){
                                    flag++;
                                    break;
                                }
                            }
                            long downEndTime = System.currentTimeMillis();
                            if (flag==0){
                                ExceptionHandVo exceptionHandVo=new ExceptionHandVo(path,downPath,fileName+".jpg");
                                errorList.add(JSON.toJSONString(exceptionHandVo));
                                logger.info("经过20次依然失败 文件url:{},文件夹:{},文件名:{},地址:{} ",key,folder,attrJPG,urlHtml);
                            }
                            //获取结束时间
                            logger.info("文件地址:{},下载耗时:{}ms,文件夹:{}",downPath,(downEndTime - downStartTime),folder);
                        }
                    }
                });
            }catch (Exception e){
                logger.info("线程出现异常: "+key+" 文件名: "+folder+" 地址: "+urlHtml);
                e.printStackTrace();
                return;
            }
            long endTime = System.currentTimeMillis();    //获取结束时间
            logger.info("下载耗时: "+urlHtml+" 文件："+folder+"下载耗时" + (endTime - startTime) + "ms");
            return;
        }else if (page.getUrl().regex(REGEX_PAGE_URL_INFO).match()){

            String urlHtml = page.getUrl().toString();
            System.out.println("详情页当前url:"+urlHtml);
            Html html = page.getHtml();
            String name = html.xpath("//div[@id='info']/h1[1]").toString();
            String h1 = Jsoup.parse(name).getElementsByTag("h1").text();
            h1=h1.replaceAll("[/:*?<>|\\\\]","_");
            h1=h1.replaceAll(" ","");
            String path1 = fileOutPath +"/"+ h1;
            File file1=new File(path1);
            if (file1.exists()){
                existList.add("当前url已存在: "+urlHtml+" 文件名: "+h1+" 地址: "+path1);
                downLoadList.add(h1+" 当前文件已存在");
                logger.info("当前url已存在: "+urlHtml+" 文件名: "+h1+" 地址: "+path1);
                return;
            }
            String s = html.xpath("//div[@id='info']/div[1]").regex("\\d+").toString();
            int pageNum = Integer.parseInt(s);
            nameMap.put(urlHtml,h1);
            List<String> list=new ArrayList<>();
            String url=urlHtml+"list2/";
            page.addTargetRequest(url);
            return;

        }else {
            String urlHtml = page.getUrl().toString();
            logger.info("无法确定是哪个的url:"+urlHtml);
        }

    }



    public static String getContent(String rule,String text) {
        Pattern pattern = Pattern.compile(rule);
        Matcher matcher = pattern.matcher(text);
        if(matcher.find()) {
            String group=matcher.group();
            return group;
        }
        return null;
    }


    //获取HTML内容
    private static String getHtml(String url)throws Exception{
        URL url1=new URL(url);
        URLConnection connection=url1.openConnection();
        connection.setConnectTimeout(12000);
        connection.setReadTimeout(12000);
        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36)");
        InputStream in=connection.getInputStream();
        InputStreamReader isr=new InputStreamReader(in);
        BufferedReader br=new BufferedReader(isr);

        String line;
        StringBuffer sb=new StringBuffer();
        while((line=br.readLine())!=null){
            sb.append(line,0,line.length());
            sb.append('\n');
        }
        br.close();
        isr.close();
        in.close();
        return sb.toString();
    }

    //    public static void main(String[] args) {
//        boolean exist = exist("https://zh.nyahentai.xyz/parody/touhou-project/page/2");
//        System.out.println(exist);
//    }
    private static boolean exist(String url) {
        try {
            URL u = new URL(url);
            HttpURLConnection huc = (HttpURLConnection) u.openConnection();
            huc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36)");
            huc.setRequestMethod ("HEAD");
            huc.setConnectTimeout(20000); //视情况设置超时时间
            huc.connect();
            return huc.getResponseCode() == HttpURLConnection.HTTP_OK;
        } catch (Exception e) {
            return false;
        }
    }
    private static boolean existToSteam(String url) {
        try {
            URL u = new URL(url);
            InputStream inputStream = u.openStream();
            return true;
        } catch (Exception e) {
            return false;
        }
    }
    private static String existUrl(String jpg,String png,String url1) {
        String url=null;
        int a=0;
        long startTime = System.currentTimeMillis();    //获取开始时间
        for (int i=0;i<=10;i++){
            a++;
            if (exist(jpg)){
                url=jpg;
                break;
            }
            if (exist(png)){
                url=png;
                break;
            }
        }
        long endTime = System.currentTimeMillis();    //获取结束时间
        System.out.println("地址: "+url1+" 循环了 "+a+" 链接时间：" + (endTime - startTime) + "ms");
        return url;
    }


}
